diff --git a/assets/src/bundles/admin/forge-add.js b/assets/src/bundles/admin/forge-add.js
new file mode 100644
--- /dev/null
+++ b/assets/src/bundles/admin/forge-add.js
@@ -0,0 +1,395 @@
+/**
+ * Copyright (C) 2021 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
+import {handleFetchError, csrfPost, htmlAlert} from 'utils/functions';
+import {swhSpinnerSrc} from 'utils/constants';
+
+let authorizedForgeTable;
+let unauthorizedForgeTable;
+let pendingAddForgeRequestsTable;
+let acceptedAddForgeRequestsTable;
+let rejectedAddForgeRequestsTable;
+
+function enableRowSelection(tableSel) {
+ $(`${tableSel} tbody`).on('click', 'tr', function() {
+ if ($(this).hasClass('selected')) {
+ $(this).removeClass('selected');
+ $(tableSel).closest('.tab-pane').find('.swh-action-need-selection').prop('disabled', true);
+ } else {
+ $(`${tableSel} tr.selected`).removeClass('selected');
+ $(this).addClass('selected');
+ $(tableSel).closest('.tab-pane').find('.swh-action-need-selection').prop('disabled', false);
+ }
+ });
+}
+
+export function initForgeAddAdmin() {
+ $(document).ready(() => {
+
+ $.fn.dataTable.ext.errMode = 'throw';
+
+ authorizedForgeTable = $('#swh-authorized-forge-urls').DataTable({
+ serverSide: true,
+ ajax: Urls.admin_forge_add_authorized_urls_list(),
+ columns: [{data: 'url', name: 'url'}],
+ scrollY: '50vh',
+ scrollCollapse: true,
+ info: false
+ });
+ enableRowSelection('#swh-authorized-forge-urls');
+ swh.webapp.addJumpToPagePopoverToDataTable(authorizedForgeTable);
+
+ unauthorizedForgeTable = $('#swh-unauthorized-forge-urls').DataTable({
+ serverSide: true,
+ ajax: Urls.admin_forge_add_unauthorized_urls_list(),
+ columns: [{data: 'url', name: 'url'}],
+ scrollY: '50vh',
+ scrollCollapse: true,
+ info: false
+ });
+ enableRowSelection('#swh-unauthorized-forge-urls');
+ swh.webapp.addJumpToPagePopoverToDataTable(unauthorizedForgeTable);
+
+ const columnsData = [
+ {
+ data: 'id',
+ name: 'id',
+ visible: false,
+ searchable: false
+ },
+ {
+ data: 'request_date',
+ name: 'request_date',
+ render: (data, type, row) => {
+ if (type === 'display') {
+ const date = new Date(data);
+ return date.toLocaleString();
+ }
+ return data;
+ }
+ },
+ {
+ data: 'forge_type',
+ name: 'forge_type'
+ },
+ {
+ data: 'forge_url',
+ name: 'forge_url'
+ }
+ ];
+
+ pendingAddForgeRequestsTable = $('#swh-forge-add-pending-requests').DataTable({
+ serverSide: true,
+ processing: true,
+ language: {
+ processing: ` `
+ },
+ ajax: Urls.forge_add_requests_list('pending'),
+ searchDelay: 1000,
+ columns: columnsData,
+ scrollY: '50vh',
+ scrollCollapse: true,
+ order: [[0, 'desc']],
+ responsive: {
+ details: {
+ type: 'none'
+ }
+ }
+ });
+ enableRowSelection('#swh-forge-add-pending-requests');
+ swh.webapp.addJumpToPagePopoverToDataTable(pendingAddForgeRequestsTable);
+
+ columnsData.push({
+ name: 'info',
+ render: (data, type, row) => {
+ if (row.add_task_status === 'succeeded' || row.add_task_status === 'failed' ||
+ row.note != null) {
+ return ` `;
+ } else {
+ return '';
+ }
+ }
+ });
+
+ rejectedAddForgeRequestsTable = $('#swh-forge-add-rejected-requests').DataTable({
+ serverSide: true,
+ processing: true,
+ language: {
+ processing: ` `
+ },
+ ajax: Urls.forge_add_requests_list('rejected'),
+ searchDelay: 1000,
+ columns: columnsData,
+ scrollY: '50vh',
+ scrollCollapse: true,
+ order: [[0, 'desc']],
+ responsive: {
+ details: {
+ type: 'none'
+ }
+ }
+ });
+ enableRowSelection('#swh-forge-add-rejected-requests');
+ swh.webapp.addJumpToPagePopoverToDataTable(rejectedAddForgeRequestsTable);
+
+ columnsData.splice(columnsData.length - 1, 0, {
+ data: 'add_task_status',
+ name: 'add_task_status'
+ });
+
+ acceptedAddForgeRequestsTable = $('#swh-forge-add-accepted-requests').DataTable({
+ serverSide: true,
+ processing: true,
+ language: {
+ processing: ` `
+ },
+ ajax: Urls.forge_add_requests_list('accepted'),
+ searchDelay: 1000,
+ columns: columnsData,
+ scrollY: '50vh',
+ scrollCollapse: true,
+ order: [[0, 'desc']],
+ responsive: {
+ details: {
+ type: 'none'
+ }
+ }
+ });
+ enableRowSelection('#swh-forge-add-accepted-requests');
+ swh.webapp.addJumpToPagePopoverToDataTable(acceptedAddForgeRequestsTable);
+
+ $('#swh-forge-add-requests-nav-item').on('shown.bs.tab', () => {
+ pendingAddForgeRequestsTable.draw();
+ });
+
+ $('#swh-forge-add-url-filters-nav-item').on('shown.bs.tab', () => {
+ authorizedForgeTable.draw();
+ });
+
+ $('#swh-authorized-forges-tab').on('shown.bs.tab', () => {
+ authorizedForgeTable.draw();
+ });
+
+ $('#swh-unauthorized-forges-tab').on('shown.bs.tab', () => {
+ unauthorizedForgeTable.draw();
+ });
+
+ $('#swh-forge-add-requests-pending-tab').on('shown.bs.tab', () => {
+ pendingAddForgeRequestsTable.draw();
+ });
+
+ $('#swh-forge-add-requests-accepted-tab').on('shown.bs.tab', () => {
+ acceptedAddForgeRequestsTable.draw();
+ });
+
+ $('#swh-forge-add-requests-rejected-tab').on('shown.bs.tab', () => {
+ rejectedAddForgeRequestsTable.draw();
+ });
+
+ $('#swh-forge-add-requests-pending-tab').click(() => {
+ pendingAddForgeRequestsTable.ajax.reload(null, false);
+ });
+
+ $('#swh-forge-add-requests-accepted-tab').click(() => {
+ acceptedAddForgeRequestsTable.ajax.reload(null, false);
+ });
+
+ $('#swh-forge-add-requests-rejected-tab').click(() => {
+ rejectedAddForgeRequestsTable.ajax.reload(null, false);
+ });
+
+ $('body').on('click', e => {
+ if ($(e.target).parents('.popover').length > 0) {
+ e.stopPropagation();
+ } else if ($(e.target).parents('.swh-add-forge-request-info').length === 0) {
+ $('.swh-add-forge-request-info').popover('dispose');
+ }
+ });
+
+ });
+}
+
+export async function addAuthorizedForgeUrl() {
+ const forgeUrl = $('#swh-authorized-url-prefix').val();
+ const addForgeUrl = Urls.admin_forge_add_add_authorized_url(forgeUrl);
+ try {
+ const response = await csrfPost(addForgeUrl);
+ handleFetchError(response);
+ authorizedForgeTable.row.add({'url': forgeUrl}).draw();
+ $('.swh-add-authorized-forge-status').html(
+ htmlAlert('success', 'The forge url has been successfully added in the authorized list.', true)
+ );
+ } catch (_) {
+ $('.swh-add-authorized-forge-status').html(
+ htmlAlert('warning', 'The provided forge url prefix is already registered in the authorized list.', true)
+ );
+ }
+}
+
+export async function removeAuthorizedForgeUrl() {
+ const forgeUrl = $('#swh-authorized-forge-urls tr.selected').text();
+ if (forgeUrl) {
+ const removeForgeUrl = Urls.admin_forge_add_remove_authorized_url(forgeUrl);
+ try {
+ const response = await csrfPost(removeForgeUrl);
+ handleFetchError(response);
+ authorizedForgeTable.row('.selected').remove().draw();
+ } catch (_) {}
+ }
+}
+
+export async function addUnauthorizedForgeUrl() {
+ const forgeUrl = $('#swh-unauthorized-url-prefix').val();
+ const addForgeUrl = Urls.admin_forge_add_add_unauthorized_url(forgeUrl);
+ try {
+ const response = await csrfPost(addForgeUrl);
+ handleFetchError(response);
+ unauthorizedForgeTable.row.add({'url': forgeUrl}).draw();
+ $('.swh-add-unauthorized-forge-status').html(
+ htmlAlert('success', 'The forge url prefix has been successfully added in the unauthorized list.', true)
+ );
+ } catch (_) {
+ $('.swh-add-unauthorized-forge-status').html(
+ htmlAlert('warning', 'The provided forge url prefix is already registered in the unauthorized list.', true)
+ );
+ }
+}
+
+export async function removeUnauthorizedForgeUrl() {
+ const forgeUrl = $('#swh-unauthorized-forge-urls tr.selected').text();
+ if (forgeUrl) {
+ const removeForgeUrl = Urls.admin_forge_add_remove_unauthorized_url(forgeUrl);
+ try {
+ const response = await csrfPost(removeForgeUrl);
+ handleFetchError(response);
+ unauthorizedForgeTable.row('.selected').remove().draw();
+ } catch (_) {};
+ }
+}
+
+export function acceptForgeAddRequest() {
+ const selectedRow = pendingAddForgeRequestsTable.row('.selected');
+ if (selectedRow.length) {
+ const acceptForgeAddRequestCallback = async() => {
+ const rowData = selectedRow.data();
+ const acceptAddRequestUrl = Urls.admin_forge_add_request_accept(rowData['visit_type'], rowData['forge_url']);
+ await csrfPost(acceptAddRequestUrl);
+ pendingAddForgeRequestsTable.ajax.reload(null, false);
+ };
+
+ swh.webapp.showModalConfirm(
+ 'Accept forge add request ?',
+ 'Are you sure to accept this forge add request ?',
+ acceptForgeAddRequestCallback);
+ }
+}
+
+const rejectModalHtml = `
+
+`;
+
+export function rejectForgeAddRequest() {
+ const selectedRow = pendingAddForgeRequestsTable.row('.selected');
+ const rowData = selectedRow.data();
+ if (selectedRow.length) {
+ const rejectForgeAddRequestCallback = async() => {
+ $('#swh-web-modal-html').modal('hide');
+ const rejectAddRequestUrl = Urls.admin_forge_add_request_reject(
+ rowData['visit_type'], rowData['forge_url']);
+ await csrfPost(rejectAddRequestUrl, {},
+ JSON.stringify({note: $('#swh-rejection-text').val()}));
+ pendingAddForgeRequestsTable.ajax.reload(null, false);
+ };
+
+ let currentRejectionReason = 'custom';
+ const rejectionTexts = {};
+ swh.webapp.showModalHtml('Reject forge add request ?', rejectModalHtml);
+ $('#swh-rejection-reason').on('change', (event) => {
+ // backup current textarea value
+ rejectionTexts[currentRejectionReason] = $('#swh-rejection-text').val();
+ currentRejectionReason = event.target.value;
+ let newRejectionText = '';
+ if (rejectionTexts.hasOwnProperty(currentRejectionReason)) {
+ // restore previous textarea value
+ newRejectionText = rejectionTexts[currentRejectionReason];
+ } else {
+ // fill textarea with default text according to rejection type
+ if (currentRejectionReason === 'invalid-forge') {
+ newRejectionText = `The forge with URL ${rowData['forge_url']} is not ` +
+ `a link to a ${rowData['visit_type']} repository.`;
+ } else if (currentRejectionReason === 'invalid-forge-type') {
+ newRejectionText = `The forge with URL ${rowData['forge_url']} is not ` +
+ `of type ${rowData['visit_type']}.`;
+ } else if (currentRejectionReason === 'forge-not-found') {
+ newRejectionText = `The forge with URL ${rowData['forge_url']} cannot be found.`;
+ }
+ }
+ $('#swh-rejection-text').val(newRejectionText);
+ });
+ $('#swh-rejection-form').on('submit', (event) => {
+ event.preventDefault();
+ event.stopPropagation();
+ // ensure confirmation modal will be displayed above the html modal
+ $('#swh-web-modal-html').css('z-index', 4000);
+ swh.webapp.showModalConfirm(
+ 'Reject forge add request ?',
+ 'Are you sure to reject this forge add request ?',
+ rejectForgeAddRequestCallback);
+ });
+ }
+}
+
+function removeForgeAddRequest(requestTable) {
+ const selectedRow = requestTable.row('.selected');
+ if (selectedRow.length) {
+ const requestId = selectedRow.data()['id'];
+ const removeForgeAddRequestCallback = async() => {
+ const removeAddRequestUrl = Urls.admin_forge_add_request_remove(requestId);
+ await csrfPost(removeAddRequestUrl);
+ requestTable.ajax.reload(null, false);
+ };
+
+ swh.webapp.showModalConfirm(
+ 'Remove forge add request ?',
+ 'Are you sure to remove this forge add request ?',
+ removeForgeAddRequestCallback);
+ }
+}
+
+export function removePendingForgeAddRequest() {
+ removeForgeAddRequest(pendingAddForgeRequestsTable);
+}
+
+export function removeAcceptedForgeAddRequest() {
+ removeForgeAddRequest(acceptedAddForgeRequestsTable);
+}
+
+export function removeRejectedForgeAddRequest() {
+ removeForgeAddRequest(rejectedAddForgeRequestsTable);
+}
diff --git a/assets/src/bundles/admin/index.js b/assets/src/bundles/admin/index.js
--- a/assets/src/bundles/admin/index.js
+++ b/assets/src/bundles/admin/index.js
@@ -1,5 +1,5 @@
/**
- * Copyright (C) 2018 The Software Heritage developers
+ * Copyright (C) 2018-2021 The Software Heritage developers
* See the AUTHORS file at the top-level directory of this distribution
* License: GNU Affero General Public License version 3, or any later version
* See top-level LICENSE file for more information
@@ -7,3 +7,4 @@
export * from './deposit';
export * from './origin-save';
+export * from './forge-add';
diff --git a/assets/src/bundles/forge_add/index.js b/assets/src/bundles/forge_add/index.js
new file mode 100644
--- /dev/null
+++ b/assets/src/bundles/forge_add/index.js
@@ -0,0 +1,427 @@
+/**
+ * Copyright (C) 2021 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
+import {csrfPost, handleFetchError, htmlAlert, removeUrlFragment} from 'utils/functions';
+import {swhSpinnerSrc} from 'utils/constants';
+
+let addForgeRequestsTable;
+
+async function forgeAddRequest(
+ forgeType, forgeUrl, acceptedCallback, pendingCallback, errorCallback
+) {
+ // Actually trigger the forge add request
+ const addForgeUrlRequestUrl = Urls.api_1_add_forge(forgeType, forgeUrl);
+
+ // show computation spinner
+ $('.swh-processing-forge-add-request').css('display', 'block');
+ const headers = {};
+ const body = null;
+
+ try {
+ const response = await csrfPost(addForgeUrlRequestUrl, headers, body);
+ handleFetchError(response);
+ const data = await response.json();
+ // hide the computation spinner
+ $('.swh-processing-forge-add-request').css('display', 'none');
+ if (data.add_forge_request_status === 'accepted') {
+ acceptedCallback();
+ } else {
+ pendingCallback();
+ }
+ } catch (response) {
+ $('.swh-processing-forge-add-request').css('display', 'none');
+ const errorData = await response.json();
+ errorCallback(response.status, errorData);
+ };
+}
+
+const userRequestsFilterCheckbox = `
+
+
+
+ show only your own requests
+
+
+`;
+
+export function initForgeAdd() {
+
+ $(document).ready(() => {
+
+ $.fn.dataTable.ext.errMode = 'none';
+
+ // set git as the default value as before
+ $('#swh-input-forge-type').val('gitlab');
+
+ addForgeRequestsTable = $('#swh-forge-add-requests')
+ .on('error.dt', (e, settings, techNote, message) => {
+ $('#swh-forge-add-request-list-error').text(
+ 'An error occurred while retrieving the add forge requests list'
+ );
+ console.log(message);
+ })
+ .DataTable({
+ serverSide: true,
+ processing: true,
+ language: {
+ processing: ` `
+ },
+ ajax: {
+ url: Urls.forge_add_requests_list('all'),
+ data: (d) => {
+ if (swh.webapp.isUserLoggedIn() && $('#swh-add-forge-requests-user-filter').prop('checked')) {
+ d.user_requests_only = '1';
+ }
+ }
+ },
+ searchDelay: 1000,
+ dom: '<"row"<"col-sm-3"l><"col-sm-6 text-left user-requests-filter"><"col-sm-3"f>>' +
+ '<"row"<"col-sm-12"tr>>' +
+ '<"row"<"col-sm-5"i><"col-sm-7"p>>',
+ fnInitComplete: function() {
+ if (swh.webapp.isUserLoggedIn()) {
+ $('div.user-requests-filter').html(userRequestsFilterCheckbox);
+ $('#swh-add-forge-requests-user-filter').on('change', () => {
+ addForgeRequestsTable.draw();
+ });
+ }
+ },
+ columns: [
+ {
+ data: 'request_date',
+ name: 'request_date',
+ render: (data, type, row) => {
+ if (type === 'display') {
+ const date = new Date(data);
+ return date.toLocaleString();
+ }
+ return data;
+ }
+ },
+ {
+ data: 'forge_type',
+ name: 'forge_type'
+ },
+ {
+ data: 'forge_url',
+ name: 'forge_url'
+ },
+ {
+ data: 'request_status',
+ name: 'request_status'
+ },
+ {
+ data: 'task_status',
+ name: 'task_status'
+ },
+ {
+ name: 'info',
+ render: (data, type, row) => {
+ if (row.task_status === 'succeeded' || row.task_status === 'failed' ||
+ row.note != null) {
+ return ` `;
+ } else {
+ return '';
+ }
+ }
+ }
+ ],
+ scrollY: '50vh',
+ scrollCollapse: true,
+ order: [[0, 'desc']],
+ responsive: {
+ details: {
+ type: 'none'
+ }
+ }
+ });
+
+ swh.webapp.addJumpToPagePopoverToDataTable(addForgeRequestsTable);
+
+ $('#swh-forge-add-requests-list').on('shown.bs.tab', () => {
+ addForgeRequestsTable.draw();
+ window.location.hash = '#forge-requests';
+ });
+
+ $('#swh-forge-add-request-help').on('shown.bs.tab', () => {
+ removeUrlFragment();
+ $('.swh-add-forge-request-info').popover('dispose');
+ });
+
+ const addForgeRequestAcceptedAlert = htmlAlert(
+ 'success',
+ 'The "add forge now" request has been accepted and will be processed as soon as possible.',
+ true
+ );
+
+ const addForgeRequestPendingAlert = htmlAlert(
+ 'warning',
+ 'The "add forge now" request has been put in pending state and may be accepted for processing after manual review.',
+ true
+ );
+
+ const addForgeRequestRateLimitedAlert = htmlAlert(
+ 'danger',
+ 'The rate limit for "add forge now" requests has been reached. Please try again later.',
+ true
+ );
+
+ const addForgeRequestUnknownErrorAlert = htmlAlert(
+ 'danger',
+ 'An unexpected error happened when submitting the "add forge now request".',
+ true
+ );
+
+ $('#swh-add-forge-forge-form').submit(async event => {
+ event.preventDefault();
+ event.stopPropagation();
+ $('.alert').alert('close');
+ if (event.target.checkValidity()) {
+ $(event.target).removeClass('was-validated');
+ const forgeType = $('#swh-input-visit-type').val();
+ const forgeUrl = $('#swh-input-forge-url').val();
+
+ forgeAddRequest(
+ forgeType,
+ forgeUrl,
+ () => $('#swh-forge-add-request-status').html(addForgeRequestAcceptedAlert),
+ () => $('#swh-forge-add-request-status').html(addForgeRequestPendingAlert),
+ (statusCode, errorData) => {
+ $('#swh-forge-add-request-status').css('color', 'red');
+ if (statusCode === 403) {
+ const errorAlert = htmlAlert('danger', `Error: ${errorData['reason']}`);
+ $('#swh-forge-add-request-status').html(errorAlert);
+ } else if (statusCode === 429) {
+ $('#swh-forge-add-request-status').html(addForgeRequestRateLimitedAlert);
+ } else if (statusCode === 400) {
+ const errorAlert = htmlAlert('danger', errorData['reason']);
+ $('#swh-forge-add-request-status').html(errorAlert);
+ } else {
+ $('#swh-forge-add-request-status').html(addForgeRequestUnknownErrorAlert);
+ }
+ });
+ } else {
+ $(event.target).addClass('was-validated');
+ }
+ });
+
+ $('#swh-show-forge-add-requests-list').on('click', (event) => {
+ event.preventDefault();
+ $('.nav-tabs a[href="#swh-forge-add-requests-list"]').tab('show');
+ });
+
+ $('#swh-input-forge-url').on('input', function(event) {
+ const forgeUrl = $(this).val().trim();
+ $(this).val(forgeUrl);
+ $('#swh-input-visit-type option').each(function() {
+ const val = $(this).val();
+ if (val && forgeUrl.includes(val)) {
+ $(this).prop('selected', true);
+ }
+ });
+ });
+
+ if (window.location.hash === '#forge-requests') {
+ $('.nav-tabs a[href="#swh-forge-add-requests-list"]').tab('show');
+ }
+
+ });
+
+}
+
+export function validateAddForgeUrl(input) {
+ let forgeUrl = null;
+ let validUrl = true;
+
+ try {
+ forgeUrl = new URL(input.value.trim());
+ } catch (TypeError) {
+ validUrl = false;
+ }
+
+ if (validUrl) {
+ const allowedProtocols = ['http:', 'https:'];
+ validUrl = (
+ allowedProtocols.find(protocol => protocol === forgeUrl.protocol) !== undefined
+ );
+ }
+
+ if (validUrl) {
+ input.setCustomValidity('');
+ } else {
+ input.setCustomValidity('The forge url is not valid or does not reference a code repository');
+ }
+}
+
+export function initTakeNewSnapshot() {
+
+ const newSnapshotRequestAcceptedAlert = htmlAlert(
+ 'success',
+ 'The "take new snapshot" request has been accepted and will be processed as soon as possible.',
+ true
+ );
+
+ const newSnapshotRequestPendingAlert = htmlAlert(
+ 'warning',
+ 'The "take new snapshot" request has been put in pending state and may be accepted for processing after manual review.',
+ true
+ );
+
+ const newSnapshotRequestRateLimitAlert = htmlAlert(
+ 'danger',
+ 'The rate limit for "take new snapshot" requests has been reached. Please try again later.',
+ true
+ );
+
+ const newSnapshotRequestUnknownErrorAlert = htmlAlert(
+ 'danger',
+ 'An unexpected error happened when submitting the "add forge now request".',
+ true
+ );
+
+ $(document).ready(() => {
+ $('#swh-take-new-snapshot-form').submit(event => {
+ event.preventDefault();
+ event.stopPropagation();
+
+ const forgeType = $('#swh-input-visit-type').val();
+ const forgeUrl = $('#swh-input-forge-url').val();
+
+ forgeAddRequest(forgeType, forgeUrl,
+ () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert),
+ () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert),
+ (statusCode, errorData) => {
+ $('#swh-take-new-snapshot-request-status').css('color', 'red');
+ if (statusCode === 403) {
+ const errorAlert = htmlAlert('danger', `Error: ${errorData['detail']}`, true);
+ $('#swh-take-new-snapshot-request-status').html(errorAlert);
+ } else if (statusCode === 429) {
+ $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestRateLimitAlert);
+ } else {
+ $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestUnknownErrorAlert);
+ }
+ });
+ });
+ });
+}
+
+export function formatValuePerType(type, value) {
+ // Given some typed value, format and return accordingly formatted value
+ const mapFormatPerTypeFn = {
+ 'json': (v) => JSON.stringify(v, null, 2),
+ 'date': (v) => new Date(v).toLocaleString(),
+ 'raw': (v) => v,
+ 'duration': (v) => v + ' seconds'
+ };
+
+ return value === null ? null : mapFormatPerTypeFn[type](value);
+}
+
+export async function displayForgeAddRequestInfo(event, addForgeRequestId) {
+ event.stopPropagation();
+ const addForgeRequestTaskInfoUrl = Urls.forge_add_task_info(addForgeRequestId);
+ // close popover when clicking again on the info icon
+ if ($(event.target).data('bs.popover')) {
+ $(event.target).popover('dispose');
+ return;
+ }
+ $('.swh-add-forge-request-info').popover('dispose');
+ $(event.target).popover({
+ animation: false,
+ boundary: 'viewport',
+ container: 'body',
+ title: 'Add request task information ' +
+ ' `,
+ content: `
+
+
+
Fetching task information ...
+
+
`,
+ html: true,
+ placement: 'left',
+ sanitizeFn: swh.webapp.filterXSS
+ });
+
+ $(event.target).on('shown.bs.popover', function() {
+ const popoverId = $(this).attr('aria-describedby');
+ $(`#${popoverId} .mdi-close`).click(() => {
+ $(this).popover('dispose');
+ });
+ });
+
+ $(event.target).popover('show');
+ const response = await fetch(addForgeRequestTaskInfoUrl);
+ const addForgeRequestTaskInfo = await response.json();
+
+ let content;
+ if ($.isEmptyObject(addForgeRequestTaskInfo)) {
+ content = 'Not available';
+
+ } else if (addForgeRequestTaskInfo.note != null) {
+ content = addForgeRequestTaskInfo.note;
+ } else {
+ const addForgeRequestInfo = [];
+ const taskData = {
+ 'Type': ['raw', 'forge_type'],
+ 'Arguments': ['json', 'arguments'],
+ 'Id': ['raw', 'id'],
+ 'Scheduling date': ['date', 'scheduled'],
+ 'Start date': ['date', 'started'],
+ 'Completion date': ['date', 'ended'],
+ 'Duration': ['duration', 'duration'],
+ 'Runner': ['raw', 'worker'],
+ 'Log': ['raw', 'message']
+ };
+ for (const [title, [type, property]] of Object.entries(taskData)) {
+ if (addForgeRequestTaskInfo.hasOwnProperty(property)) {
+ addForgeRequestInfo.push({
+ key: title,
+ value: formatValuePerType(type, addForgeRequestTaskInfo[property])
+ });
+ }
+ }
+ content = '';
+ for (const info of addForgeRequestInfo) {
+ content +=
+ `
+ ${info.key}
+
+ ${info.value}
+
+ `;
+ }
+ content += '
';
+ }
+ $('.swh-popover').html(content);
+ $(event.target).popover('update');
+}
+
+export function fillForgeAddRequestFormAndScroll(visitType, forgeUrl) {
+ $('#swh-input-forge-url').val(forgeUrl);
+ let forgeTypeFound = false;
+ $('#swh-input-visit-type option').each(function() {
+ const val = $(this).val();
+ if (val && forgeUrl.includes(val)) {
+ $(this).prop('selected', true);
+ forgeTypeFound = true;
+ }
+ });
+ if (!forgeTypeFound) {
+ $('#swh-input-visit-type option').each(function() {
+ const val = $(this).val();
+ if (val === visitType) {
+ $(this).prop('selected', true);
+ }
+ });
+ }
+ window.scrollTo(0, 0);
+}
diff --git a/swh/web/admin/forge_add.py b/swh/web/admin/forge_add.py
new file mode 100644
--- /dev/null
+++ b/swh/web/admin/forge_add.py
@@ -0,0 +1,220 @@
+# Copyright (C) 2021 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import json
+
+from django.conf import settings
+from django.contrib.admin.views.decorators import staff_member_required
+from django.core.exceptions import ObjectDoesNotExist
+from django.core.paginator import Paginator
+from django.http import HttpResponse, JsonResponse
+from django.shortcuts import render
+from django.views.decorators.http import require_POST
+
+from swh.web.admin.adminurls import admin_route
+from swh.web.common.models import (
+ ADD_FORGE_REQUEST_PENDING,
+ ADD_FORGE_REQUEST_REJECTED,
+ AddAuthorizedForge,
+ AddForgeRequest,
+ AddUnauthorizedForge,
+)
+
+
+def create_add_forge_request(forge_type: str, forge_url: str) -> None:
+ """Schedule a listing forge task.
+
+ """
+ pass
+
+
+@admin_route(r"forge/add/", view_name="admin-forge-add")
+@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
+def _admin_forge_add(request):
+ return render(request, "admin/forge-add.html")
+
+
+def _datatables_forge_urls_response(request, urls_query_set):
+ search_value = request.GET["search[value]"]
+ if search_value:
+ urls_query_set = urls_query_set.filter(url__icontains=search_value)
+
+ column_order = request.GET["order[0][column]"]
+ field_order = request.GET["columns[%s][name]" % column_order]
+ order_dir = request.GET["order[0][dir]"]
+ if order_dir == "desc":
+ field_order = "-" + field_order
+
+ urls_query_set = urls_query_set.order_by(field_order)
+
+ table_data = {}
+ table_data["draw"] = int(request.GET["draw"])
+ table_data["recordsTotal"] = urls_query_set.count()
+ table_data["recordsFiltered"] = urls_query_set.count()
+ length = int(request.GET["length"])
+ page = int(request.GET["start"]) / length + 1
+ paginator = Paginator(urls_query_set, length)
+ urls_query_set = paginator.page(page).object_list
+ table_data["data"] = [{"url": u.url} for u in urls_query_set]
+ return JsonResponse(table_data)
+
+
+@admin_route(
+ r"forge/add/authorized_urls/list/",
+ view_name="admin-forge-add-authorized-urls-list",
+)
+@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
+def _admin_forge_add_authorized_urls_list(request):
+ authorized_urls = AddAuthorizedForge.objects.all()
+ return _datatables_forge_urls_response(request, authorized_urls)
+
+
+@admin_route(
+ r"forge/add/authorized_urls/add/(?P.+)/",
+ view_name="admin-forge-add-add-authorized-url",
+)
+@require_POST
+@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
+def _admin_forge_add_add_authorized_url(request, forge_url):
+ try:
+ AddAuthorizedForge.objects.get(url=forge_url)
+ except ObjectDoesNotExist:
+ # add the new authorized url
+ AddAuthorizedForge.objects.create(url=forge_url)
+ # check if pending add requests with that url prefix exist
+ pending_add_requests = AddForgeRequest.objects.filter(
+ forge_url__startswith=forge_url, request_status=ADD_FORGE_REQUEST_PENDING
+ )
+ # create forge add tasks for previously pending requests
+ for psr in pending_add_requests:
+ create_add_forge_request(psr.forge_type, psr.forge_url)
+ status_code = 200
+ else:
+ status_code = 400
+ return HttpResponse(status=status_code)
+
+
+@admin_route(
+ r"forge/add/authorized_urls/remove/(?P.+)/",
+ view_name="admin-forge-add-remove-authorized-url",
+)
+@require_POST
+@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
+def _admin_forge_add_remove_authorized_url(request, forge_url):
+ try:
+ entry = AddAuthorizedForge.objects.get(url=forge_url)
+ except ObjectDoesNotExist:
+ status_code = 404
+ else:
+ entry.delete()
+ status_code = 200
+ return HttpResponse(status=status_code)
+
+
+@admin_route(
+ r"forge/add/unauthorized_urls/list/",
+ view_name="admin-forge-add-unauthorized-urls-list",
+)
+@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
+def _admin_forge_add_unauthorized_urls_list(request):
+ unauthorized_urls = AddUnauthorizedForge.objects.all()
+ return _datatables_forge_urls_response(request, unauthorized_urls)
+
+
+@admin_route(
+ r"forge/add/unauthorized_urls/add/(?P.+)/",
+ view_name="admin-forge-add-add-unauthorized-url",
+)
+@require_POST
+@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
+def _admin_forge_add_add_unauthorized_url(request, forge_url):
+ try:
+ AddUnauthorizedForge.objects.get(url=forge_url)
+ except ObjectDoesNotExist:
+ AddUnauthorizedForge.objects.create(url=forge_url)
+ # check if pending add requests with that url prefix exist
+ pending_add_requests = AddForgeRequest.objects.filter(
+ forge_url__startswith=forge_url, request_status=ADD_FORGE_REQUEST_PENDING
+ )
+ # mark pending requests as rejected
+ for psr in pending_add_requests:
+ psr.status = ADD_FORGE_REQUEST_REJECTED
+ psr.add()
+ status_code = 200
+ else:
+ status_code = 400
+ return HttpResponse(status=status_code)
+
+
+@admin_route(
+ r"forge/add/unauthorized_urls/remove/(?P.+)/",
+ view_name="admin-forge-add-remove-unauthorized-url",
+)
+@require_POST
+@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
+def _admin_forge_add_remove_unauthorized_url(request, forge_url):
+ try:
+ entry = AddUnauthorizedForge.objects.get(url=forge_url)
+ except ObjectDoesNotExist:
+ status_code = 404
+ else:
+ entry.delete()
+ status_code = 200
+ return HttpResponse(status=status_code)
+
+
+@admin_route(
+ r"forge/add/request/accept/(?P.+)/url/(?P.+)/",
+ view_name="admin-forge-add-request-accept",
+)
+@require_POST
+@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
+def _admin_forge_add_request_accept(request, forge_type, forge_url):
+ try:
+ AddAuthorizedForge.objects.get(url=forge_url)
+ except ObjectDoesNotExist:
+ AddAuthorizedForge.objects.create(url=forge_url)
+ create_add_forge_request(forge_type, forge_url)
+ return HttpResponse(status=200)
+
+
+@admin_route(
+ r"forge/add/request/reject/(?P.+)/url/(?P.+)/",
+ view_name="admin-forge-add-request-reject",
+)
+@require_POST
+@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
+def _admin_forge_add_request_reject(request, forge_type, forge_url):
+ try:
+ AddUnauthorizedForge.objects.get(url=forge_url)
+ except ObjectDoesNotExist:
+ AddUnauthorizedForge.objects.create(url=forge_url)
+ sor = AddForgeRequest.objects.get(
+ forge_type=forge_type,
+ forge_url=forge_url,
+ request_status=ADD_FORGE_REQUEST_PENDING,
+ )
+
+ sor.status = ADD_FORGE_REQUEST_REJECTED
+ sor.note = json.loads(request.body).get("note")
+ sor.add()
+ return HttpResponse(status=200)
+
+
+@admin_route(
+ r"forge/add/request/remove/(?P.+)/",
+ view_name="admin-forge-add-request-remove",
+)
+@require_POST
+@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
+def _admin_forge_add_request_remove(request, sor_id):
+ try:
+ entry = AddForgeRequest.objects.get(id=sor_id)
+ except ObjectDoesNotExist:
+ status_code = 404
+ else:
+ entry.delete()
+ status_code = 200
+ return HttpResponse(status=status_code)
diff --git a/swh/web/admin/urls.py b/swh/web/admin/urls.py
--- a/swh/web/admin/urls.py
+++ b/swh/web/admin/urls.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2018 The Software Heritage developers
+# Copyright (C) 2018-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -9,6 +9,7 @@
from swh.web.admin.adminurls import AdminUrls
import swh.web.admin.deposit # noqa
+import swh.web.admin.forge_add # noqa
import swh.web.admin.origin_save # noqa
diff --git a/swh/web/api/views/forge_add.py b/swh/web/api/views/forge_add.py
new file mode 100644
--- /dev/null
+++ b/swh/web/api/views/forge_add.py
@@ -0,0 +1,467 @@
+# Copyright (C) 2021 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import os
+from typing import Any, Dict, List, Optional
+
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
+from django.core.validators import URLValidator
+from django.db.models import QuerySet
+from django.utils.html import escape
+
+from swh.scheduler.utils import create_oneshot_task_dict
+from swh.web.api.apidoc import api_doc, format_docstring
+from swh.web.api.apiurls import api_route
+from swh.web.auth.utils import (
+ API_ADD_FORGE_PERMISSION,
+ SWH_AMBASSADOR_PERMISSION,
+ privileged_user,
+)
+from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc
+from swh.web.common.forge_add import get_forge_types
+from swh.web.common.models import (
+ ADD_FORGE_REQUEST_ACCEPTED,
+ ADD_FORGE_REQUEST_PENDING,
+ ADD_FORGE_REQUEST_REJECTED,
+ AddAuthorizedForge,
+ AddForgeRequest,
+ AddUnauthorizedForge,
+)
+from swh.web.common.typing import AddForgeRequestInfo
+from swh.web.config import scheduler
+
+_validate_url = URLValidator(schemes=["http", "https"])
+
+
+def get_forge_add_authorized_urls() -> List[str]:
+ """Get the list of forge url prefixes authorized to be immediately loaded into the
+ archive (whitelist).
+
+ Returns:
+ The list of authorized forge url prefixes
+
+ """
+ return [forge.url for forge in AddAuthorizedForge.objects.all()]
+
+
+def get_forge_add_unauthorized_urls() -> List[str]:
+ """Get the list of forge url prefixes forbidden to be loaded into the archive
+ (blacklist).
+
+ Returns:
+ the list of unauthorized forge url prefixes
+
+ """
+ return [forge.url for forge in AddUnauthorizedForge.objects.all()]
+
+
+def can_add_forge(forge_url: str, bypass_pending_review: bool = False) -> str:
+ """str if a software forge can be added into the archive.
+
+ Based on the forge url, the add request will be either:
+
+ * immediately accepted if the url is whitelisted
+ * rejected if the url is blacklisted
+ * put in pending state for manual review otherwise
+
+ Args:
+ forge_url (str): the software forge url to check
+
+ Returns:
+ str: the forge add request request_status, either **accepted**,
+ **rejected** or **pending**
+
+ """
+ # forge url may be blacklisted
+ for url_prefix in get_forge_add_unauthorized_urls():
+ if forge_url.startswith(url_prefix):
+ return ADD_FORGE_REQUEST_REJECTED
+
+ # if the forge url is in the white list, it can be immediately added
+ for url_prefix in get_forge_add_authorized_urls():
+ if forge_url.startswith(url_prefix):
+ return ADD_FORGE_REQUEST_ACCEPTED
+
+ # otherwise, the forge url needs to be manually verified if the user
+ # that submitted it does not have special permission
+ if bypass_pending_review:
+ # mark the forge URL as trusted in that case
+ AddAuthorizedForge.objects.get_or_create(url=forge_url)
+ return ADD_FORGE_REQUEST_ACCEPTED
+ else:
+ return ADD_FORGE_REQUEST_PENDING
+
+
+def _check_forge_type(forge_type: str, privileged_user: bool = False) -> None:
+ visit_type_tasks = get_forge_types(privileged_user)
+ if forge_type not in visit_type_tasks:
+ allowed_visit_types = ", ".join(visit_type_tasks)
+ raise BadInputExc(
+ f"Visit of type {forge_type} cannot be added! "
+ f"Allowed types are the following: {allowed_visit_types}"
+ )
+
+
+def _check_forge_url_valid(forge_url: str) -> None:
+ """Check the forge url is valid and raise if not."""
+ try:
+ _validate_url(forge_url)
+ except ValidationError:
+ raise BadInputExc(f"The provided forge url ({escape(forge_url)}) is not valid!")
+
+
+def _update_add_forge_request_info(
+ add_forge_request: AddForgeRequest,
+ task: Optional[Dict[str, Any]] = None,
+ task_run: Optional[Dict[str, Any]] = None,
+) -> AddForgeRequestInfo:
+ """Update add forge request information out of the task and task_run information
+
+ Args:
+ add_forge_request: Add request
+ task: Associated scheduler task information about the add request
+ task_run: Most recent run occurrence of the associated task
+
+ Returns:
+ Summary of the add request information updated.
+
+ """
+
+ # FIXME: Actually determine what'add_forge_request the update condition
+ must_add = False
+ if not add_forge_request.task_date or not add_forge_request.task_status:
+ if task:
+ must_add = True
+ add_forge_request.request_status = task["status"]
+ if task_run:
+ must_add = True
+ add_forge_request.task_date = task_run["ended"]
+
+ if must_add:
+ add_forge_request.save()
+
+ return add_forge_request.to_dict()
+
+
+def create_add_forge_request(
+ forge_type: str,
+ forge_url: str,
+ privileged_user: bool = False,
+ user_id: Optional[int] = None,
+) -> AddForgeRequestInfo:
+ """Create a loading task to add a software forge into the archive.
+
+ This function aims to create a software forge listing task trough the use of the
+ swh-scheduler component.
+
+ First, some checks are performed to see if the forge type and forge url are valid
+ but also if the the add request can be accepted. If those checks passed, the listing
+ task is then created. Otherwise, the add request is put in pending or rejected
+ state.
+
+ All the submitted add requests are logged into the swh-web database to keep track of
+ them.
+
+ Args:
+ forge_type: the type of forge to list (e.g. gitlab, cgit, ...)
+ forge_url: the url of the forge to add
+ privileged: Whether the user has some more privilege than other (bypass
+ review, access to privileged other forge types)
+ user_id: User identifier (provided when authenticated)
+
+ Raises:
+ BadInputExc: the forge type or forge url is invalid or inexistent
+ ForbiddenExc: the provided forge url is blacklisted
+
+ Returns:
+ dict: A dict describing the add request with the following keys:
+
+ * **forge_type**: the type of visit to perform
+ * **forge_url**: the url of the forge
+ * **add_request_date**: the date the request was submitted
+ * **add_forge_request_status**: the request request_status, either
+ ****accepted**, rejected** or **pending**
+ * **add_task_status**: the forge loading task request_status, either
+ **not created**, **not yet scheduled**, **scheduled**,
+ **succeed** or **failed**
+
+ """
+ # FIXME: actually read this out of the scheduler *somehow*
+ forge_type_tasks = {
+ "sourceforge": "list-sourceforge-incremental",
+ "opam": "list-opam",
+ "gitlab": "list-gitlab-incremental",
+ "cgit": "list-cgit",
+ "launchpad": "list-launchpad-incremental",
+ }
+ _check_forge_type(forge_type, privileged_user)
+ _check_forge_url_valid(forge_url)
+
+ print(f"################### forge_url {forge_url}")
+ print(f"################### forge_type {forge_type}")
+ # if all checks passed so far, we can try and add the forge
+ add_forge_request_status = can_add_forge(forge_url, privileged_user)
+ print(f"################### add_forge_request_status {add_forge_request_status}")
+ task = None
+
+ # if the forge add request is accepted, create a scheduler
+ # task to load it into the archive
+ if add_forge_request_status == ADD_FORGE_REQUEST_ACCEPTED:
+ # create a task with high priority
+ task_kwargs: Dict[str, Any] = {
+ "url": forge_url,
+ }
+
+ add_forge_request = None
+ # get list of previously submitted add requests (most recent first)
+ current_add_forge_requests = list(
+ AddForgeRequest.objects.filter(
+ forge_type=forge_type, forge_url=forge_url
+ ).order_by("-request_date")
+ )
+
+ # if no add requests already submitted, create the scheduler task
+ if not current_add_forge_requests:
+ task_dict = create_oneshot_task_dict(
+ forge_type_tasks[forge_type], **task_kwargs
+ )
+
+ task = scheduler().create_tasks([task_dict])[0]
+
+ # pending add request has been accepted
+ if add_forge_request:
+ add_forge_request.request_status = ADD_FORGE_REQUEST_ACCEPTED
+ add_forge_request.task_id = task["id"]
+ add_forge_request.save()
+ else:
+ add_forge_request = AddForgeRequest.objects.create(
+ forge_type=forge_type,
+ forge_url=forge_url,
+ request_status=add_forge_request_status,
+ task_id=task["id"],
+ user_ids=f'"{user_id}"' if user_id else None,
+ )
+
+ # add request must be manually reviewed for acceptation
+ elif add_forge_request_status == ADD_FORGE_REQUEST_PENDING:
+ # check if there is already such a add request already submitted,
+ # no need to add it to the database in that case
+ try:
+ add_forge_request = AddForgeRequest.objects.get(
+ forge_type=forge_type,
+ forge_url=forge_url,
+ request_status=add_forge_request_status,
+ )
+ user_ids = (
+ add_forge_request.user_ids
+ if add_forge_request.user_ids is not None
+ else ""
+ )
+ if user_id is not None and f'"{user_id}"' not in user_ids:
+ # update user ids list
+ add_forge_request.user_ids = f'{add_forge_request.user_ids},"{user_id}"'
+ add_forge_request.save()
+
+ # if not add it to the database
+ except ObjectDoesNotExist:
+ add_forge_request = AddForgeRequest.objects.create(
+ forge_type=forge_type,
+ forge_url=forge_url,
+ request_status=add_forge_request_status,
+ user_ids=f'"{user_id}"' if user_id else None,
+ )
+ # forge cannot be added as its url is blacklisted,
+ # log the request to the database anyway
+ else:
+ add_forge_request = AddForgeRequest.objects.create(
+ forge_type=forge_type,
+ forge_url=forge_url,
+ request_status=add_forge_request_status,
+ user_ids=f'"{user_id}"' if user_id else None,
+ )
+
+ if add_forge_request_status == ADD_FORGE_REQUEST_REJECTED:
+ raise ForbiddenExc(
+ (
+ 'The "add forge now" request has been rejected '
+ "because the provided forge url is blacklisted."
+ )
+ )
+
+ assert add_forge_request is not None
+ return _update_add_forge_request_info(add_forge_request, task)
+
+
+def _savable_forge_types():
+ docstring = ""
+ if os.environ.get("DJANGO_SETTINGS_MODULE") != "swh.web.settings.tests":
+ forge_types = sorted(get_forge_types())
+ docstring = ""
+ for forge_type in forge_types[:-1]:
+ docstring += f"**{forge_type}**, "
+ docstring += f"and **{forge_types[-1]}**"
+ return docstring
+
+
+def update_add_forge_requests_from_queryset(
+ requests_queryset: QuerySet,
+) -> List[AddForgeRequestInfo]:
+ """Update all add requests from a AddForgeRequest queryset, update their status in db
+ and return the list of impacted add_requests.
+
+ Args:
+ requests_queryset: input AddForgeRequest queryset
+
+ Returns:
+ A list of add forge request info dicts
+
+ """
+ task_ids = []
+ for add_forge_query in requests_queryset:
+ task_ids.append(add_forge_query.task_id)
+ add_requests = []
+ if task_ids:
+ try:
+ tasks = scheduler().get_tasks(task_ids)
+ tasks = {task["id"]: task for task in tasks}
+ task_runs = scheduler().get_task_runs(tasks)
+ task_runs = {task_run["task"]: task_run for task_run in task_runs}
+ except Exception:
+ # allow to avoid mocking api GET responses for /forge/add endpoint when
+ # running cypress tests as scheduler is not available
+ tasks = {}
+ task_runs = {}
+ for add_forge_query in requests_queryset:
+ sr_dict = _update_add_forge_request_info(
+ add_forge_query,
+ tasks.get(add_forge_query.task_id),
+ task_runs.get(add_forge_query.task_id),
+ )
+ add_requests.append(sr_dict)
+ return add_requests
+
+
+def get_add_forge_requests(
+ forge_type: str, forge_url: str
+) -> List[AddForgeRequestInfo]:
+ """
+ Get all add requests for a given software forge.
+
+ Args:
+ forge_type: the type of visit
+ forge_url: the url of the forge
+
+ Raises:
+ BadInputExc: the visit type or forge url is invalid
+ swh.web.common.exc.NotFoundExc: no add requests can be found for the
+ given forge
+
+ Returns:
+ list: A list of add forge requests dict as described in
+ :func:`swh.web.common.forge_add.create_add_forge_request`
+ """
+ _check_forge_type(forge_type)
+ _check_forge_url_valid(forge_url)
+ forges_query_set = AddForgeRequest.objects.filter(
+ forge_type=forge_type, forge_url=forge_url
+ )
+ if forges_query_set.count() == 0:
+ raise NotFoundExc(
+ f"No add forge requests found on forge with url {forge_url}"
+ f"and type {forge_type}."
+ )
+ return update_add_forge_requests_from_queryset(forges_query_set)
+
+
+@api_route(
+ r"/forge/add/(?P.+)/url/(?P.+)/",
+ "api-1-add-forge",
+ methods=["GET", "POST"],
+ throttle_scope="swh_add_forge",
+ never_cache=True,
+)
+@api_doc("/forge/add/")
+@format_docstring(forge_types=_savable_forge_types())
+def api_add_forge(
+ request, forge_type: str, forge_url: str
+) -> List[AddForgeRequestInfo]:
+ """.. http:get:: /api/1/forge/add/(forge_type)/url/(forge_url)/
+ .. http:post:: /api/1/forge/add/(forge_type)/url/(forge_url)/
+
+ Request adding a new software forge to list into the archive or check the
+ request_status of previously created add requests.
+
+ That endpoint enables creating an add forge task for listing through a POST
+ request.
+
+ Depending on the provided forge url, the add request can either be:
+
+ * immediately **accepted**, for well known and supported forge type hosting
+ providers like for instance GitLab or Heptapod instance
+ * **rejected**, in case the url is blacklisted by Software Heritage
+ * **put in pending state** until a manual check is done in order to
+ determine if it can be listed or not
+
+ Once a saving request has been accepted, its associated task request_status can
+ then be checked through a GET request on the same url. Returned request_status
+ can either be:
+
+ * **not created**: no adding task has been created
+ * **not yet scheduled**: adding task has been created but its execution has
+ not yet been scheduled
+ * **scheduled**: the task execution has been scheduled
+ * **succeeded**: the task has been successfully executed
+ * **failed**: the task has been executed but it failed
+
+ When issuing a POST request an object will be returned while a GET request will
+ return an array of objects (as multiple add requests might have been submitted
+ for the same forge).
+
+ :param string forge_type: the type of forge to list (currently the supported
+ types are {forge_types})
+
+ :param string forge_url: the url of the forge to add
+
+ {common_headers}
+
+ :>json string forge_url: the url of the forge to add
+ :>json string forge_type: the type of forge
+ :>json string request_date: the date (in iso format) the add
+ request was issued
+ :>json string request_status: the request_status of the add request,
+ either **accepted**, **rejected** or **pending**
+ :>json string task_status: the request_status of the forge saving task,
+ either **not created**, **not yet scheduled**, **scheduled**,
+ **succeeded** or **failed**
+ :>json string note: optional note giving details about the add request,
+ for instance why it has been rejected
+
+ :statuscode 200: no error
+ :statuscode 400: an invalid forge type or forge url has been provided
+ :statuscode 403: the provided forge url is blacklisted
+ :statuscode 404: no add requests have been found for a given forge
+
+ """
+
+ if request.method == "POST":
+ add_forge_request = create_add_forge_request(
+ forge_type,
+ forge_url,
+ privileged_user(
+ request,
+ permissions=[SWH_AMBASSADOR_PERMISSION, API_ADD_FORGE_PERMISSION],
+ ),
+ user_id=request.user.id,
+ )
+ # FIXME: drop the id of the result
+ # add_forge_request.pop("id", None)
+ result = [add_forge_request]
+ else:
+ add_forge_requests = get_add_forge_requests(forge_type, forge_url)
+ # FIXME: drop the id of the elements result
+ result = [add_forge_request for add_forge_request in add_forge_requests]
+
+ print(f"############ result {result}")
+ return result
diff --git a/swh/web/auth/utils.py b/swh/web/auth/utils.py
--- a/swh/web/auth/utils.py
+++ b/swh/web/auth/utils.py
@@ -17,6 +17,7 @@
SWH_AMBASSADOR_PERMISSION = "swh.ambassador"
API_SAVE_ORIGIN_PERMISSION = "swh.web.api.save_origin"
+API_ADD_FORGE_PERMISSION = "swh.web.api.add_forge"
ADMIN_LIST_DEPOSIT_PERMISSION = "swh.web.admin.list_deposits"
diff --git a/swh/web/browse/urls.py b/swh/web/browse/urls.py
--- a/swh/web/browse/urls.py
+++ b/swh/web/browse/urls.py
@@ -46,6 +46,10 @@
return redirect(reverse("origin-save"))
+def _browse_forge_add_view(request):
+ return redirect(reverse("forge-add"))
+
+
urlpatterns = [
url(r"^$", _browse_search_view),
url(r"^help/$", _browse_help_view, name="browse-help"),
@@ -58,6 +62,7 @@
swhid_browse,
name="browse-swhid",
),
+ url(r"^forge/add/$", _browse_forge_add_view, name="browse-forge-add"),
]
urlpatterns += BrowseUrls.get_url_patterns()
diff --git a/swh/web/common/forge_add.py b/swh/web/common/forge_add.py
new file mode 100644
--- /dev/null
+++ b/swh/web/common/forge_add.py
@@ -0,0 +1,22 @@
+# Copyright (C) 2021 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from typing import List
+
+
+def get_forge_types(privileged_user: bool = False) -> List[str]:
+ """Retrieve the allowed list of forge types users can request.
+
+ Args:
+ privileged_user: Whether the user is privileged (True) or not (False).
+
+ Returns:
+ The list of supported forge to request for listing.
+
+ """
+ # FIXME: Determine where to store such information
+ hardcoded_types = ["gitlab", "heptapod", "cgit", "launchpad", "opam"]
+ hardcoded_types.sort()
+ return hardcoded_types
diff --git a/swh/web/common/migrations/0013_auto_20211119_1031.py b/swh/web/common/migrations/0013_auto_20211119_1031.py
new file mode 100644
--- /dev/null
+++ b/swh/web/common/migrations/0013_auto_20211119_1031.py
@@ -0,0 +1,108 @@
+# Generated by Django 2.2.24 on 2021-11-19 10:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("swh_web_common", "0012_saveoriginrequest_note"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="AddAuthorizedForge",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("url", models.CharField(max_length=200)),
+ ],
+ options={"db_table": "add_authorized_forge",},
+ ),
+ migrations.CreateModel(
+ name="AddForgeRequest",
+ fields=[
+ ("id", models.BigAutoField(primary_key=True, serialize=False)),
+ ("request_date", models.DateTimeField(auto_now_add=True)),
+ ("forge_type", models.CharField(max_length=200)),
+ ("forge_url", models.CharField(max_length=200)),
+ (
+ "request_status",
+ models.TextField(
+ choices=[
+ ("accepted", "accepted"),
+ ("rejected", "rejected"),
+ ("pending", "pending"),
+ ],
+ default="pending",
+ ),
+ ),
+ ("user_ids", models.TextField(null=True)),
+ ("note", models.TextField(null=True)),
+ ],
+ options={"db_table": "add_forge_request", "ordering": ["-id"],},
+ ),
+ migrations.CreateModel(
+ name="AddUnauthorizedForge",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("url", models.CharField(max_length=200)),
+ ],
+ options={"db_table": "add_unauthorized_forge",},
+ ),
+ migrations.AddIndex(
+ model_name="addunauthorizedforge",
+ index=models.Index(fields=["url"], name="add_unautho_url_c89686_idx"),
+ ),
+ migrations.AddIndex(
+ model_name="addforgerequest",
+ index=models.Index(
+ fields=["forge_url", "request_status"],
+ name="add_forge_r_forge_u_3703b5_idx",
+ ),
+ ),
+ migrations.AddIndex(
+ model_name="addauthorizedforge",
+ index=models.Index(fields=["url"], name="add_authori_url_dd17ed_idx"),
+ ),
+ migrations.AddField(
+ model_name="addforgerequest",
+ name="task_date",
+ field=models.DateTimeField(null=True),
+ ),
+ migrations.AddField(
+ model_name="addforgerequest",
+ name="task_id",
+ field=models.IntegerField(default=-1),
+ ),
+ migrations.AddField(
+ model_name="addforgerequest",
+ name="task_status",
+ field=models.TextField(
+ choices=[
+ ("not created", "not created"),
+ ("not yet scheduled", "not yet scheduled"),
+ ("scheduled", "scheduled"),
+ ("succeeded", "succeeded"),
+ ("failed", "failed"),
+ ("running", "running"),
+ ],
+ default="not created",
+ ),
+ ),
+ ]
diff --git a/swh/web/common/models.py b/swh/web/common/models.py
--- a/swh/web/common/models.py
+++ b/swh/web/common/models.py
@@ -5,7 +5,7 @@
from django.db import models
-from swh.web.common.typing import SaveOriginRequestInfo
+from swh.web.common.typing import AddForgeRequestInfo, SaveOriginRequestInfo
class SaveAuthorizedOrigin(models.Model):
@@ -133,3 +133,106 @@
def __str__(self) -> str:
return str(self.to_dict())
+
+
+class AddAuthorizedForge(models.Model):
+ """Model table holding authorized forge urls to be loaded into the archive.
+
+ """
+
+ url = models.CharField(max_length=200, null=False)
+
+ class Meta:
+ app_label = "swh_web_common"
+ db_table = "add_authorized_forge"
+ indexes = [models.Index(fields=["url"])]
+
+ def __str__(self):
+ return self.url
+
+
+class AddUnauthorizedForge(models.Model):
+ """Model table holding unauthorized forge urls to be loaded into the archive.
+
+ """
+
+ url = models.CharField(max_length=200, null=False)
+
+ class Meta:
+ app_label = "swh_web_common"
+ db_table = "add_unauthorized_forge"
+ indexes = [models.Index(fields=["url"])]
+
+ def __str__(self):
+ return self.url
+
+
+ADD_FORGE_REQUEST_ACCEPTED = "accepted"
+ADD_FORGE_REQUEST_REJECTED = "rejected"
+ADD_FORGE_REQUEST_PENDING = "pending"
+
+ADD_FORGE_REQUEST_STATUS = [
+ (ADD_FORGE_REQUEST_ACCEPTED, ADD_FORGE_REQUEST_ACCEPTED),
+ (ADD_FORGE_REQUEST_REJECTED, ADD_FORGE_REQUEST_REJECTED),
+ (ADD_FORGE_REQUEST_PENDING, ADD_FORGE_REQUEST_PENDING),
+]
+
+ADD_FORGE_TASK_NOT_CREATED = "not created"
+ADD_FORGE_TASK_NOT_YET_SCHEDULED = "not yet scheduled"
+ADD_FORGE_TASK_SCHEDULED = "scheduled"
+ADD_FORGE_TASK_SUCCEEDED = "succeeded"
+ADD_FORGE_TASK_FAILED = "failed"
+ADD_FORGE_TASK_RUNNING = "running"
+
+ADD_FORGE_TASK_STATUS = SAVE_TASK_STATUS
+
+
+class AddForgeRequest(models.Model):
+ """The "add forge" users issued requests table.
+
+ """
+
+ id = models.BigAutoField(primary_key=True)
+ request_date = models.DateTimeField(auto_now_add=True)
+ forge_type = models.CharField(max_length=200, null=False)
+ forge_url = models.CharField(max_length=200, null=False)
+ request_status = models.TextField(
+ choices=ADD_FORGE_REQUEST_STATUS, default=ADD_FORGE_REQUEST_PENDING
+ )
+ task_id = models.IntegerField(default=-1)
+ task_date = models.DateTimeField(null=True)
+ task_status = models.TextField(
+ choices=ADD_FORGE_TASK_STATUS, default=ADD_FORGE_TASK_NOT_CREATED
+ )
+
+ # store ids of users that submitted the request as string list
+ user_ids = models.TextField(null=True)
+ note = models.TextField(null=True)
+
+ class Meta:
+ app_label = "swh_web_common"
+ db_table = "add_forge_request"
+ ordering = ["-id"]
+ indexes = [models.Index(fields=["forge_url", "request_status"])]
+
+ def to_dict(self) -> AddForgeRequestInfo:
+ """Map the request save model object to a json serializable dict.
+
+ Returns:
+ The corresponding AddForgeRequestInfo json serializable dict.
+
+ """
+ return AddForgeRequestInfo(
+ id=self.id,
+ forge_url=self.forge_url,
+ forge_type=self.forge_type,
+ request_date=self.request_date.isoformat(),
+ request_status=self.request_status,
+ task_id=self.task_id,
+ task_date=self.task_date.isoformat() if self.task_date else None,
+ task_status=self.task_status,
+ note=self.note,
+ )
+
+ def __str__(self) -> str:
+ return str(self.to_dict())
diff --git a/swh/web/common/typing.py b/swh/web/common/typing.py
--- a/swh/web/common/typing.py
+++ b/swh/web/common/typing.py
@@ -261,3 +261,24 @@
"""content length of the artifact"""
last_modified: Optional[str]
"""Last modification time reported by the server (as iso8601 string)"""
+
+
+class AddForgeRequestInfo(TypedDict):
+ id: int
+ """Unique key"""
+ request_date: str
+ """Date of the creation request"""
+ forge_type: str
+ """Type of the forge"""
+ request_status: Optional[str]
+ """Status of the request"""
+ forge_url: str
+ """Forge to list"""
+ task_id: Optional[int]
+ """Identifier of the loading task in the scheduler if scheduled"""
+ task_date: Optional[str]
+ """End of the listing if terminated"""
+ task_status: Optional[str]
+ """Status of the scheduled task"""
+ note: Optional[str]
+ """Optional note associated to the request, for instance rejection reason"""
diff --git a/swh/web/misc/forge_add.py b/swh/web/misc/forge_add.py
new file mode 100644
--- /dev/null
+++ b/swh/web/misc/forge_add.py
@@ -0,0 +1,114 @@
+# Copyright (C) 2021 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from typing import Any, Dict
+
+from django.conf.urls import url
+from django.core.paginator import Paginator
+from django.db.models import Q
+from django.http import JsonResponse
+from django.shortcuts import render
+
+from swh.web.auth.utils import SWH_AMBASSADOR_PERMISSION, privileged_user
+from swh.web.common.forge_add import get_forge_types
+from swh.web.common.models import AddForgeRequest
+
+
+def get_add_forge_task_info(
+ add_forge_request_id: int, full_info: bool = True
+) -> Dict[str, Any]:
+ """Get detailed information about an accepted add forge request and its associated
+ loading task.
+
+ """
+ return {}
+
+
+def _forge_add_view(request):
+ return render(
+ request,
+ "misc/forge-add.html",
+ {
+ "heading": (
+ "Request adding a software forge to list and ingest in the archive"
+ ),
+ "forge_types": get_forge_types(
+ privileged_user(request, permissions=[SWH_AMBASSADOR_PERMISSION])
+ ),
+ },
+ )
+
+
+def _forge_add_requests_list(request, status: str) -> JsonResponse:
+
+ if status != "all":
+ add_requests = AddForgeRequest.objects.filter(request_status=status)
+ else:
+ add_requests = AddForgeRequest.objects.all()
+
+ print("########### request", request)
+ print("########### status", status)
+ print("########### add_requests", add_requests)
+ table_data = {}
+ table_data["recordsTotal"] = add_requests.count()
+ table_data["draw"] = int(request.GET["draw"])
+
+ search_value = request.GET["search[value]"]
+
+ column_order = request.GET["order[0][column]"]
+ field_order = request.GET["columns[%s][name]" % column_order]
+ order_dir = request.GET["order[0][dir]"]
+ if order_dir == "desc":
+ field_order = "-" + field_order
+
+ add_requests = add_requests.order_by(field_order)
+
+ length = int(request.GET["length"])
+ page = int(int(request.GET["start"]) / length + 1)
+
+ if search_value:
+ add_requests = add_requests.filter(
+ Q(status__icontains=search_value)
+ | Q(task_status__icontains=search_value)
+ | Q(forge_type__icontains=search_value)
+ | Q(forge_url__icontains=search_value)
+ )
+
+ if (
+ int(request.GET.get("user_requests_only", "0"))
+ and request.user.is_authenticated
+ ):
+ add_requests = add_requests.filter(user_ids__contains=f'"{request.user.id}"')
+
+ table_data["recordsFiltered"] = add_requests.count()
+ paginator = Paginator(add_requests, length)
+ data = [sor.to_dict() for sor in paginator.page(page).object_list]
+ table_data["data"] = data # type: ignore
+ return JsonResponse(table_data)
+
+
+def _add_forge_task_info(request, add_request_id):
+ request_info = get_add_forge_task_info(
+ add_request_id, full_info=request.user.is_staff
+ )
+ for date_field in ("scheduled", "started", "ended"):
+ if date_field in request_info and request_info[date_field] is not None:
+ request_info[date_field] = request_info[date_field].isoformat()
+ return JsonResponse(request_info)
+
+
+urlpatterns = [
+ url(r"^add/$", _forge_add_view, name="forge-add"),
+ url(
+ r"^add/requests/list/(?P.+)/$",
+ _forge_add_requests_list,
+ name="forge-add-requests-list",
+ ),
+ url(
+ r"^add/task/info/(?P.+)/",
+ _add_forge_task_info,
+ name="forge-add-task-info",
+ ),
+]
diff --git a/swh/web/misc/urls.py b/swh/web/misc/urls.py
--- a/swh/web/misc/urls.py
+++ b/swh/web/misc/urls.py
@@ -52,6 +52,7 @@
url(r"^", include("swh.web.misc.badges")),
url(r"^metrics/prometheus/$", prometheus_metrics, name="metrics-prometheus"),
url(r"^", include("swh.web.misc.iframe")),
+ url(r"^", include("swh.web.misc.forge_add")),
]
diff --git a/swh/web/templates/admin/forge-add.html b/swh/web/templates/admin/forge-add.html
new file mode 100644
--- /dev/null
+++ b/swh/web/templates/admin/forge-add.html
@@ -0,0 +1,186 @@
+{% extends "layout.html" %}
+
+{% comment %}
+Copyright (C) 2021 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
+{% load swh_templatetags %}
+{% load render_bundle from webpack_loader %}
+
+{% block header %}
+{{ block.super }}
+{% render_bundle 'admin' %}
+{% render_bundle 'forge_add' %}
+{% endblock %}
+
+{% block title %} "Add forge" administration {% endblock %}
+
+{% block navbar-content %}
+Add forge administration
+{% endblock %}
+
+{% block content %}
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Date
+ Type
+ Url
+
+
+
+
+
+
+
+ Accept selected
+
+
+ Reject selected
+
+
+ Remove selected
+
+
+
+
+
+
+
+
+ Date
+ Type
+ Url
+ Status
+ Info
+
+
+
+
+
+
+
+ Remove selected
+
+
+
+
+
+
+
+
+ Date
+ Type
+ Url
+ Info
+
+
+
+
+
+
+
+ Remove selected
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Remove selected
+
+
+
+
+
+
+
+
+
+
+
+ Remove selected
+
+
+
+
+
+
+
+
+
+
+{% endblock %}
diff --git a/swh/web/templates/layout.html b/swh/web/templates/layout.html
--- a/swh/web/templates/layout.html
+++ b/swh/web/templates/layout.html
@@ -208,6 +208,12 @@
Save code now
+
+
+
+ Add forge now
+
+
{% if user.is_authenticated and user.is_staff or "swh.ambassador" in user.get_all_permissions %}
@@ -227,6 +233,12 @@
Save code now
+
+
+
+ Add forge now
+
+
{% endif %}
diff --git a/swh/web/templates/misc/forge-add.html b/swh/web/templates/misc/forge-add.html
new file mode 100644
--- /dev/null
+++ b/swh/web/templates/misc/forge-add.html
@@ -0,0 +1,126 @@
+{% extends "../layout.html" %}
+
+{% comment %}
+Copyright (C) 2021 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
+{% load render_bundle from webpack_loader %}
+{% load static %}
+
+{% block title %}{{ heading }} – Software Heritage archive{% endblock %}
+
+{% block header %}
+{% render_bundle 'forge_add' %}
+{% endblock %}
+
+{% block navbar-content %}
+Add forge now
+{% endblock %}
+
+{% block content %}
+
+
+ You can contribute to extend the content of the Software Heritage archive by
+ submitting a new supported forge type in our recurrent listing processes.
+ You can also contribute one origin at a time .
+
+ To submit a new supported forge type, fill the required info in the form below:
+
+
+
+ {% csrf_token %}
+
+
+
+
+
Processing "forge add now" request ...
+
+
+
+
+
+
+
+
+
A "Add forge now" request takes the following parameters:
+
+ Forge type: the type of supported forge the software archive is able to list.
+ Currently, the supported types are:
+
+ cgit
, for cgit forges
+ gitlab
, for gitlab forges
+ heptapod
, for heptapod forges
+ ...
+
+
+ Forge url: the url of the remote forge to list.
+
+
+
+ Once submitted, your "add forge" request can either be:
+
+
+ accepted: a listing of the provided forge will then be scheduled by Software Heritage in order to
+ load its content into the archive as soon as possible
+ rejected: the provided forge url is blacklisted and no listing will be scheduled
+ put in pending state: a manual review will then be performed in order to determine if the
+ forge can be safely listed
+
+
+ Once a forge request has been accepted, you can follow its current status in the
+ submitted forge add requests list .
+
+
+
+
+
+
+
+
+ Date
+ Type
+ Url
+ Request
+ Status
+ Info
+
+
+
+
+
+
+
+
+
+{% endblock %}
diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html
--- a/swh/web/templates/misc/origin-save.html
+++ b/swh/web/templates/misc/origin-save.html
@@ -23,8 +23,10 @@
{% block content %}
- You can contribute to extend the content of the Software Heritage archive by submitting an origin
- save request. To do so, fill the required info in the form below:
+ You can contribute to extend the content of the Software Heritage archive by
+ submitting an origin save request.
+ You can also contribute adding one forge .
+ To submit an origin save request, fill the required info in the form below: